import pymongo
import requests
from lxml import etree

class DouBan250:
    # 待爬取的页面地址列表
    url_list = []
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36',
        'referer': 'https://movie.douban.com/',
    }

    def __init__(self):
        """
        初始化方法（爬虫执行前的准备工作）
        1、生成要抓取的url地址
        2、连接到mongodb数据库
        """
        # 生成要抓取的url地址
        for i in range(10):
            url = f'https://movie.douban.com/top250?start={i*25}'
            self.url_list.append(url)
        # 连接到mongodb数据库
            client = pymongo.MongoClient(host='localhost', port=27017)
            self.db = client.python

    def main(self):
        """爬虫运行主入口"""
        for url in self.url_list:
            # 发送请求，获取响应
            response = self.send_request(url)
            # 提取数据
            datas = self.paser_date(response)
            # 保存数据
            self.save_data(datas)

    def send_request(self, url):
        """发送请求，获取响应"""
        response = requests.get(url, headers=self.headers)
        return response

    def paser_date(self,response):
        """解的网页面源码"""
        # 获取网页面源码
        page = response.content.decode()
        # 转换为html元素对象
        html = etree.HTML(page)
        # 定位所有内容的盒子
        li_boxes = html.xpath("//ol/li")
        # 判断盒子数量
        if len(li_boxes) == 0:
            return None
        else:
            print(li_boxes)
            data_list = []
            for li in li_boxes:
                # 提取评分
                # 这里最前面有一个.表示当前的li元素对象
                score = li.xpath('.//span[@class="rating_num"]/text()')[0]
                # 提取电影名称
                title = li.xpath('.//span[@class="title"]/text()')[0]
                # 提取导演及主演信息
                name = li.xpath('.//div[@class="bd"]/p/text()')[0]
                name = name.strip()
                name = name.replace('\xa0\xa0\xa0', ' ')
                # 提取图片地址
                img_url = li.xpath('.//div[@class="pic"]//img/@src')[0]
                data = {
                    "名称": title,
                    "评分": score,
                    "导演及主演": name,
                    "海报地址": img_url
                }
                data_list.append(data)
            return data_list


    def save_data(self, datas):
        """保存数据"""
        if datas:
            for data in datas:
                self.db.douban.insert_one(data)

if __name__ == '__main__':
    douban = DouBan250()
    douban.main()
